use DBI;
use Encode;
use Net::FTP::Recursive;
use Benchmark;


our $HOMEDIR;
our $SITECFG = "$HOMEDIR/etc/WORKING.HOST";
our $removeold;

require "veryftp_config.pl";
require "veryftp_db.pl";
require "veryftp_process.pl";
require "veryftp_util.pl";
require "dumpvar.pl";

our ($site, $site_ID, %ftp_site, @sites);   #All attributes of a ftp site
our ($filecount, $rescount);   #Number of files

####################################################################################

open_db();

my @dirs = get_dirs();  #Get a list of directories to run spider on
my $prets = new Benchmark;


foreach my $dir_ref (@dirs){
        my ($list_site, $down_site) = @$dir_ref{'list_site', 'down_site'};
        my $site = $list_site->{'site_dns_name'};
        my $port = $list_site->{'site_port'};
        my $dir_id = $dir_ref->{'dir_id'};
        my $dir = $dir_ref->{'dir_path'};
        my $list_user = $list_site->{'site_username'};
        
        #create log directory under $HOMEDIR/logs
        open(LOG, ">>"."$HOMEDIR/logs/"."$site".".log") or die "Cannot creat file $HOMEDIR/logs/$site.log\n";
        #next if $site !~ /112\.225/;
        
        #Get file list under the directory $dir
        my $filename = get_site_list($site, $port, $list_site->{'site_username'}, $list_site->{'site_password'}, $dir);
        #print "$filename\n";
        next unless $filename;
        #next if $filename !~ /172\.16\.3\.79/;

        
        #Here,check if there's a file named $filename.old,
        #if true,it's the first time we have run the spider so we just fetch all files
        #else we compare and update.
        $first_run = "yes";

        open(IN, $filename) || die "Can not open file $filename\n";
        if(-s "$filename.old"){
                open(OLDIN, "$filename.old") or die "Can not open file $filename.old\n";
                $first_run = "no";
        }

        open(LOGERR, ">$HOMEDIR/logs/$site"."-op_err") or die "Can not open error log file.\n";

        process_site_dir($dir_ref, $list_site, $down_site, $first_run);

        #To the end of the file of directories
        if (eof IN){
                close(IN);
                close(OLDIN) if($first_run eq "no");
                close(LOGERR);
                close(LOG);

                #remove old file.
                if($removeold){
                        File::copy("$filename.old", "$HOMEDIR/backup/"."$site".'-'."$cur_timestamp") if(-s $site.".tmp.old");
                        unlink("$filename.old") if(-e "$filename.old");
                        rename($filename, "$filename.old");
                }
        }
}
#Print time consumed and files updated.
my $postts = new Benchmark;
my $diff = timediff($postts, $prets);
print "Total running comsumes ". timestr($diff)."\n";
open(LOG, ">>"."$HOMEDIR/logs/spider.log") or die "Cannot creat file $site.log\n";
print LOG "Get All in ".timestr($diff)."s.\n";

set_unavailable();

close_db();
print "******************************\n";
print "newfile:$filecount\n";

#############################################################################################

sub make_stat{
}
